package com.example.book.crawler;

import org.apache.commons.lang3.StringUtils;
import us.codecraft.webmagic.*;
import us.codecraft.webmagic.downloader.HttpClientDownloader;
import us.codecraft.webmagic.pipeline.Pipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.proxy.Proxy;
import us.codecraft.webmagic.proxy.SimpleProxyProvider;

import java.math.BigDecimal;

public class TestCrawler implements PageProcessor {

    private Site site = Site.me().setRetryTimes(3).setSleepTime(1000).setTimeOut(10000);

    @Override
    public void process(Page page) {

        boolean a = true;
        int i = 1;
        while (a) {
            //  xpath: /html/body/div/div[6]/div[1]/div[2]/ul/li[1]/a
            String catalog = page.getHtml().xpath("/html/body/div/div[6]/div[1]/div[2]/ul/li[" + i + "]/a/text()").toString();
            String href = page.getHtml().xpath("/html/body/div/div[6]/div[1]/div[2]/ul/li[" + i + "]/a/@href").toString();

            if (StringUtils.isBlank(catalog)) {
                a = false;
            }
            i++;
            System.out.println("章节：" + catalog);
            System.out.println("href：" + href);
        }
    }

    @Override
    public Site getSite() {
        return site;
    }

    public static void main(String[] args) {
        HttpClientDownloader downloader = new HttpClientDownloader();
        downloader.setProxyProvider(SimpleProxyProvider.from(
                new Proxy("106.54.122.103",8088)
        ));
        Spider.create(new TestCrawler()).setDownloader(downloader).addUrl("https://www.wmtxt.com/0/1/").addPipeline(new Pipeline() {
            @Override
            public void process(ResultItems resultItems, Task task) {
                System.out.println("res".toString());
            }
        }).thread(1).run();

    }

}
